package com.shareditor.chatbotv1;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashSet;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
//import org.wltea.analyzer.lucene.IKAnalyzer;

public class Indexer 
{
	
	public static final Charset UTF8 = Charset.forName("utf8");

	public static String hexString(byte[] b) {
		String ret = "";
		for (int i = 0; i < b.length; i++) {
			String hex = Integer.toHexString(b[i] & 0xF);
			ret += hex.toUpperCase();
		}
		return ret;
	}
	
    public static void main( String[] args ) throws IOException, NoSuchAlgorithmException
    {
    	
    	//indexDir is the directory that hosts Lucene's index files 
        /*File   indexDir = new File("D:\\luceneIndex"); 
        //dataDir is the directory that hosts the text files that to be indexed 
        File   dataDir  = new File("D:\\luceneData"); 
        Analyzer luceneAnalyzer = new StandardAnalyzer(); 
        File[] dataFiles  = dataDir.listFiles(); 
        IndexWriter indexWriter = new IndexWriter(indexDir,luceneAnalyzer,true); 
        long startTime = new Date().getTime(); 
        for(int i = 0; i < dataFiles.length; i++){ 
             if(dataFiles[i].isFile() && dataFiles[i].getName().endsWith(".txt")){
                  System.out.println("Indexing file " + dataFiles[i].getCanonicalPath()); 
                  Document document = new Document(); 
                  Reader txtReader = new FileReader(dataFiles[i]); 
                  document.add(Field.Text("path",dataFiles[i].getCanonicalPath())); 
                  document.add(Field.Text("contents",txtReader)); 
                  indexWriter.addDocument(document); 
             } 
        } 
        indexWriter.optimize(); 
        indexWriter.close(); 
        long endTime = new Date().getTime(); 
            
        System.out.println("It takes " + (endTime - startTime) 
            + " milliseconds to create index for the files in directory "
            + dataDir.getPath());        
        }*/ 
    	/*
        if (args.length != 2) {
        	System.err.println("Usage: " + Indexer.class.getSimpleName() + " corpus_path index_path");
        	System.exit(-1);
        }
        
        String corpusPath = args[0];
        String indexPath = args[1];
        */
    	File directory = new File("");//设定为当前文件夹 
    	System.out.println(directory.getAbsolutePath());
    	String corpusPath = "./target/corpus";
    	String indexPath = "./target/index";
        //Analyzer analyzer = new IKAnalyzer(true);
    	StandardAnalyzer analyzer = new StandardAnalyzer();
        IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
		iwc.setOpenMode(OpenMode.CREATE);
		//iwc.setUseCompoundFile(true);
        FSDirectory e = FSDirectory.open(Paths.get(indexPath, new String[0]));
		IndexWriter indexWriter = new IndexWriter(e, iwc);
		Boolean create =true;
		if(create) {
            iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
        } else {
            iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
        }
		BufferedReader br = new BufferedReader(new InputStreamReader(
                new FileInputStream(corpusPath), "UTF-8"));
        String line = "";
        String last = "";
        long lineNum = 0;
        MessageDigest md = MessageDigest.getInstance("MD5");
        HashSet<String> mc = new HashSet<String>();
        int dupCount = 0;
        int totalCount = 0;
        long last_t = 0;
        String deleteStr;
        while ((line = br.readLine()) != null) {
        	totalCount++;
        	if (totalCount % 15000000 == 0) {
        		System.out.println("clear set");
        		mc.clear();
        	}
        	line = line.trim();
        	
        	if (0 == line.length()) {
        		continue;
        	}
        	if(totalCount ==1) {
        		deleteStr = line;
        		System.out.println("will delete:" +  deleteStr);
        	}
        	
        	if (!last.equals("")) {
        		String pair = last + line;
        		
        		byte[] md5 = md.digest(pair.getBytes(UTF8));
        		String md5_str = hexString(md5);
        		
        		if (mc.contains(md5_str)) {
        			dupCount++;
        			continue;
        		} else {
        			mc.add(md5_str);
        		}
        		Document doc = new Document();
        		doc.add(new TextField("question", last, Store.YES));
        		doc.add(new StoredField("answer", line));
        		indexWriter.addDocument(doc);
        	}
        	last = line;
        	lineNum++;
        	if (lineNum % 100000 == 0) {
        		long t = System.currentTimeMillis();
        		System.out.println("elapse second: " + (t-last_t)/1000 + " add doc " + lineNum + " totalCount:" + totalCount + " dup:" + dupCount);
        		last_t = t;
        	}
        }
		br.close();
		
		indexWriter.forceMerge(1);
		
		Document doc = new Document();
		TextField f=new TextField("title", "title book went", Store.YES);
		f.setBoost(0.1f);
		doc.add(f);
		doc.add(new StoredField("content", "book content went-------"));
		indexWriter.addDocument(doc);

		Document doc1 = new Document();
		doc1.add(new TextField("title", "推广有什么用?", Store.YES));
		doc1.add(new StoredField("content", "比较大的作用...."));
		
		/*Document doc = new Document();
		doc.add(new TextField("question", "推广有什么用?", Store.YES));
		doc.add(new StoredField("answer", "比较大的作用...."));
		indexWriter.addDocument(doc);
		/*doc.add(new TextField("question", "怎么进行推广?", Store.YES));
		doc.add(new StoredField("answer", "按照如下方法推广...."));
		indexWriter.addDocument(doc);*/
		//indexWriter.deleteDocuments(new Term("question", "school"));//删除包含"school"的文档，其实是删除了两篇文档中的一篇。 
        indexWriter.commit();		
		indexWriter.close();
		System.out.println("game over!");

    }
}
